package com.spider.moudle.crawler.forum;

import com.alibaba.fastjson.JSONObject;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.api.realize.AbstractReplyCrawler;
import com.spider.moudle.constant.Constant;
import com.spider.moudle.entity.*;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.SettingsUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
 * Created by Linbo Hu on 2017/12/25.
 * 天府社区crawler
 */
public class TianFuCrawler extends AbstractCrawler {

    private Setting setting = SettingsUtil.getSettingBySite("TianFu");

    @Override
    public void generateTask() {
        ExecutorService executorService = null;
        try {
            ResponseEntity entity = null;
            try {
                entity = downloader.get(setting.getIndex(), true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                executorService = ExecutorServiceFactoy.buildExecutorService(setting.getTask_thread_count(), ExecutorServiceFactoy.getName(setting.getName(), setting.getIndex()), new NamedThreadFactory(setting.getName(), "生成任务"));
                Document document = entity.getDocument();
                Elements elements = document.select("table[class=\"fl_tb\"]");
                for(Element e : elements) {
                    Elements a_elements = e.select("tr td div[class=\"fl_icn_g\"] a");
                    if(!a_elements.isEmpty()) {
                        for(Element a : a_elements) {
                            String url = a.absUrl("href");
                            String text = a.select("img").attr("alt");
                            text = text.replace("论坛", "").replace(" ", "");
                            Area area = Area.search(text);
                            TianFuTaskCrawler taskCrawler = new TianFuTaskCrawler(area, url);
                            executorService.submit(taskCrawler);
                        }
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if(executorService != null) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        if (task.getTaskType() == TaskType.SPEECH) {
            JSONObject item = new JSONObject();
            SpeechBean data = new SpeechBean();
            String url = task.getUrl();
            data.setUrl(url);
            data.setArea(task.getArea());
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();
                    String title = document.title();
                    Element pageInfo = document.getElementById("postlist");;
                    Elements countInfo = pageInfo.select(".floor").get(0).select(".authi").select(".cp_pls").select("span");
                    document.select(".aimg_tip").remove();
                    int read_count = 0;
                    int reply_count = 0;
                    if (countInfo != null) {
                        String readCount = countInfo.get(2).text();
                        String replyCount = countInfo.get(5).text();
                        if ("查看:".equals(readCount)) {
                            readCount = countInfo.get(3).text();
                            replyCount = countInfo.get(6).text();
                        }
                        if (readCount != null && !"".equals(readCount)) {
                            read_count = Integer.parseInt(readCount);
                        }
                        if (replyCount != null && !"".equals(replyCount)) {
                            reply_count = Integer.parseInt(replyCount);
                        }
                    }
                    //页数
                    int page = (int) Math.ceil(((double) reply_count + 1) / 10d);
                    String id = url.substring(url.indexOf("-")+1, url.lastIndexOf("."));
                    id = id.substring(0, id.indexOf("-"));
                    data.setId(id);
                    data.setTitle(title);
                    data.setReply_count(reply_count);
                    data.setRead_count(read_count);

                    Elements contentInfo = pageInfo.select(".floor");
                    Element userInfo = contentInfo.get(0);
                    String publish_time_string = null;
                    if (userInfo.select(".authi").select(".cp_pls").select("em") != null) {
                        publish_time_string = userInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[1] + " " + userInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[2] + ":00";
                    }
                    if(null == publish_time_string) {
                        return null;
                    }
                    Date publish_time = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, publish_time_string);
                    data.setPublish_time(publish_time);
                    String publish_user_url = userInfo.select(".authi").select(".avatar").select("a").get(0).absUrl("href");
                    String publish_user_id = publish_user_url.substring(publish_user_url.lastIndexOf("-")+1, publish_user_url.lastIndexOf("."));
                    String publish_user = userInfo.select(".authi").select(".floor-profile").select("a").get(0).text();
                    String publish_user_photo = userInfo.select(".authi").select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                    if (publish_user_id != null && !"".equals(publish_user_id)) {
                        publish_user_id = publish_user_id.split("\\.")[0];
                    }
                    data.setPublish_user_id(publish_user_id);
                    data.setPublish_user(publish_user);
                    data.setPublish_user_photo(publish_user_photo);
                    data.setPublish_user_jsonid(setting.getName() + "_" + publish_user_id);
                    data.setSpider_time(new Date());
                    if (CommonUtil.isNotNull(publish_user_url)) {
                        String userKey = setting.getName() + "_" + publish_user_id;
                        Task hometask = new Task(publish_user_url, TaskType.USER, WebType.FORLUM, userKey);
                        hometask.put("publish_user", publish_user);
                        hometask.put("publish_user_photo", publish_user_photo);
                        TaskManager.add(setting.getUser_task(), hometask);
                    }
                    String content = userInfo.select(".pcb").select(".t_fsz").select(".t_f").text();
                    content = CommonUtil.filtHtml(content);
                    data.setContent(content);
                    List<String> img_urls = new ArrayList<>();
                    Elements imgElements=userInfo.select(".pcb").select(".t_fsz").select(".t_f").select("img[zoomfile]");
                    for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                        Element imgElement=imgElements.get(imgIndex);
                        String img_url=imgElement.absUrl("zoomfile");
                        img_urls.add(img_url);
                    }
                    data.setImg_urls(img_urls);
                    int like_count = 0;
                    int dislike_count = 0;
                    if (userInfo.getElementById("recommendv_add") != null) {
                        like_count = Integer.parseInt(userInfo.getElementById("recommendv_add").text());
                    }
                    if (userInfo.getElementById("recommendv_subtract") != null) {
                        dislike_count = Integer.parseInt(userInfo.getElementById("recommendv_subtract").text());
                    }
                    data.setLike_count(like_count);
                    data.setDislike_count(dislike_count);
                    List<SpeechBean> reply_nodes = new ArrayList<>();
                    //将本页面的回复添加上去
                    if (!contentInfo.isEmpty()) {
                        for (int i = 1; i < contentInfo.size(); i++) {
                            SpeechBean reply = new SpeechBean();
                            Element replyInfo = contentInfo.get(i);
                            String reply_id = replyInfo.select("table").get(0).attr("id").substring(3);
                            String reply_user_home_url = replyInfo.select(".authi").select(".avatar").select("a").get(0).attr("href");
                            String reply_user_id = replyInfo.select(".authi").select(".avatar").select("a").get(0).attr("href").split("//")[1].split("-")[2].split("\\.")[0];
                            String reply_user_photo = replyInfo.select(".authi").select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                            String reply_publish_user = replyInfo.select(".authi").select(".floor-profile").select("a").get(0).text();
                            if (replyInfo.select(".authi").select(".cp_pls").select("em") != null) {
                                String reply_publish_time_string = replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[1] + " " + replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[2] + ":00";
                                Date reply_publish_time = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_publish_time_string);
                                reply.setPublish_time(reply_publish_time);
                            }
                            String homeUrl = "http://bbs.scol.com.cn/forum.php?mod=viewthread&tid=" + id + "&page=1#pid" + reply_id;
                            if (CommonUtil.isNotNull(reply_user_home_url)) {
                                String userKey = setting.getName() + "_" + reply_user_id;
                                Task hometask = new Task(reply_user_home_url, TaskType.USER, WebType.FORLUM, userKey);
                                hometask.put("publish_user", reply_user_id);
                                hometask.put("publish_user_photo", reply_user_photo);
                                TaskManager.add(setting.getUser_task(), hometask);
                            }
                            String reply_content = replyInfo.select(".pcb").select(".t_fsz").select(".t_f").text();
                            List<String> reply_img_urls = new ArrayList<>();
                            Elements replyImgElements=replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select("img[zoomfile]");
                            for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                Element imgElement=replyImgElements.get(imgIndex);
                                String img_url=imgElement.absUrl("zoomfile");
                                reply_img_urls.add(img_url);
                            }
                            reply.setUrl(homeUrl);
                            reply.setId(reply_id);
                            reply.setPublish_user_id(reply_user_id);
                            reply.setPublish_user_jsonid(setting.getName() + "_" + reply_user_id);
                            reply.setPublish_user_photo(reply_user_photo);
                            reply.setPublish_user(reply_publish_user);
                            reply.setContent(reply_content);
                            reply.setImg_urls(reply_img_urls);
                            reply.setParent_id(id);
                            reply.setAncestor_id(id);
                            reply.setArea(task.getArea());
                            reply_nodes.add(reply);
                        }
                    }
                    if (page >= setting.getMax_page_count()) {
                        page = setting.getMax_page_count();
                    }
                    if (page >= 2) {
                        int threadNum = setting.getReply_thread_count();
                        List<FutureTask<List<SpeechBean>>> futureTasks = new ArrayList<>();
                        ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, page, threadNum);
                        ExecutorService executorService = null;
                        try {
                            executorService = ExecutorServiceFactoy.buildExecutorService(threadNum, ExecutorServiceFactoy.getName(setting.getName(), url), new NamedThreadFactory(setting.getName(), "回复"));
                            for (int i = 0; i < splitPages.size(); i++) {
                                TianFuReplyCrwler replyCrwler = new TianFuReplyCrwler(splitPages.get(i), url, id, id, task.getArea());
                                FutureTask<List<SpeechBean>> futureTask = new FutureTask<>(replyCrwler);
                                futureTasks.add(futureTask);
                                executorService.submit(futureTask);
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                            log.error(setting.getName() + "获取回复线程执行异常");
                        } finally {
                            if (null != executorService) {
                                executorService.shutdown();
                            }
                        }
                        for (FutureTask<List<SpeechBean>> futureTask : futureTasks) {
                            List<SpeechBean> list = futureTask.get();
                            reply_nodes.addAll(list);
                        }
                        Collections.sort(reply_nodes, (o1, o2) -> {
                            Date s1 = o1.getPublish_time();
                            Date s2 = o2.getPublish_time();
                            int flag = s1.compareTo(s2);
                            return flag;
                        });
                        data.setReply_nodes(reply_nodes);
                    }
                    item.put("data", data.toJSONObject());
                    return item;
                } catch (Exception e) {
                    log.error(setting.getName() + "获取言论数据解析错误: " + url);
                }
            }
        } else if (task.getTaskType() == TaskType.USER) {
            String userKey = task.getTaskKey();
            if (CommonUtil.isNull(userKey)) {
                System.err.println("User key is not empty where task type belong to home page");
                return null;
            }
            JSONObject item = new JSONObject();
            UserBean data;
            String url = task.getUrl();
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url, false);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    data = new UserBean();
                    data.setUrl(url);
                    Document document = entity.getDocument();
                    Elements info_elements = document.select("div[class=\"bm_c u_profile\"]");
                    if(!info_elements.isEmpty()) {
                        Element info_element = info_elements.get(0);
                        String publish_user_id = userKey.substring(userKey.indexOf("_") + 1);
                        data.setPublish_user_id(publish_user_id);
                        String publish_user = (String) task.get("publish_user");
                        data.setPublish_user(publish_user);
                        String publish_user_photo = (String) task.get("publish_user_photo");
                        data.setPublish_user_photo(publish_user_photo);
                        data.setJson_id(userKey);
                        Elements count_element = info_element.select("ul[class=\"cl bbda pbm mbm\"] a");
                        for(Element a: count_element) {
                            String text = a.text();
                            if(text.contains("好友数")) {
                                text = text.replace("好友数", "").trim();
                                if(CommonUtil.isNotNull(text)) {
                                    data.setFans_count(Integer.parseInt(text));
                                }

                            } else if(text.contains("回帖数")) {
                                text = text.replace("回帖数", "").trim();
                                data.setReply_article_count(Integer.parseInt(text));
                            } else if(text.contains("主题数")) {
                                text = text.replace("主题数", "").trim();
                                data.setArticle_count(Integer.parseInt(text));
                            }
                        }
                        Elements register_time_elements = info_element.select("ul[class=\"pf_l\"] li");
                        String register_time_string = register_time_elements.get(1).text();
                        if(!register_time_string.contains("最后访问")) {
                            register_time_string = register_time_elements.get(0).text();
                        }
                        register_time_string = register_time_string.replace("注册时间", "");
                        register_time_string = register_time_string + ":00";
                        Date register_time = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, register_time_string);
                        data.setRegister_time(register_time);
                    }
                    item.put("data", data.toJSONObject());
                    return item;
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error(e.getMessage(), e);
                }
            }
        }
        return null;
    }


    public String getRealUrl(String script){
        String new_script = script;
        String[] strs = {"window","location","'assign'","'href'","'replace'"};
        List<String> equal = new ArrayList<String>();
        for(int i = 0;i<strs.length;i++) {
            String regex = String.format("[_A-Za-z0-9 =]+%s;",strs[i]);
            Pattern p = Pattern.compile(regex);
            Matcher m = p.matcher(new_script);
            while (m.find()) {
                equal.add(m.group(0));
            }

        }
        if(equal.size()>0) {
            for (String s : equal) {
                String varname = s.split("=")[0].trim();
                new_script = new_script.replaceAll(s,"");
                String var = s.split("=")[1];
                var = var.substring(0,var.length()-1);
                new_script = new_script.replaceAll(varname,var);
            }
        }

        List<String> equal1 = new ArrayList<>();
        for(int i = 0;i<strs.length;i++){
            String regex1 = String.format("\\[\\s*%s\\]",strs[i]);
            String new_replace = String.format(".%s",strs[i]);
            Pattern p = Pattern.compile(regex1);
            Matcher m = p.matcher(new_script);
            while (m.find()) {
                equal1.add(m.group(0));
            }

            new_replace = new_replace.replaceAll("\'","");
            for (String s : equal1) {
                new_script = new_script.replace(s,new_replace);
            }
        }

        new_script = new_script.replaceAll("location\\.href=","var url=");
        new_script = new_script.replaceAll("window\\.href=","var url2=");
        new_script = new_script.replaceAll("location\\.assign","var url=");
        new_script = new_script.replaceAll("location\\.replace","var url=");
        new_script = new_script.replaceAll("location=","var url=");
        new_script = new_script.replaceAll("return getName\\(\\)","return ''");
        ScriptEngineManager manager = new ScriptEngineManager();
        ScriptEngine engine = manager.getEngineByName("javascript");
        String url = "";
        try{
            url= (String) engine.eval(new_script+";function rtn(){return url;}rtn();");
            System.out.println(url);
            // engine.eval("alert(\"js alert\");");    // 不能调用浏览器中定义的js函数 // 错误，会抛出alert引用不存在的异常
        }catch(ScriptException e) {

            e.printStackTrace();
        }
        String url2 = "";
        if(url!=null){
            url2 = url.split("\\?")[1].split("=")[1];
        }else{
            url2 = "";
        }
        return url2;
    }

    /**
     * 天府社区生成任务线程
     */
    private class TianFuTaskCrawler implements Runnable {

        /**
         * 哪个地区
         */
        private Area area;

        /**
         * url
         */
        private String url;

        private Downloader downloader = DownloaderFactory.buildDownloader();

        public TianFuTaskCrawler(Area area, String url) {
            this.area = area;
            this.url = url;
        }

        @Override
        public void run() {
            for (int page = 0; page < 5; page++) {
                //是否抓下一页
                boolean haveLater = false;
                try {
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(url, true);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements tr_elements = document.select("table tbody tr");
                        for(int i = 0; i < tr_elements.size(); i++) {
                            Element tr = tr_elements.get(i);
                            Elements a_elements = tr.select("a[class=\"s xst\"]");
                            if(a_elements.isEmpty()) {
                                continue;
                            }
                            Element a = a_elements.get(0);
                            String speech_url = a.absUrl("href");
                            //这个key是用来查询该言论是否被爬过的唯一标识
                            String speechkey = setting.getSite() + "_" + speech_url.substring(speech_url.lastIndexOf("/") + 1, speech_url.lastIndexOf(".")).trim();
                            Task task = new Task(speech_url, TaskType.SPEECH, WebType.FORLUM, speechkey, area);
                            TaskManager.add(setting.getSpeech_task(), task);
                            if (i == tr_elements.size() - 1) {
                                String updateTimeString = tr.select("td[class=\"by\"] span").text();
                                updateTimeString = updateTimeString + ":00";
                                Date updateTime = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, updateTimeString);
                                Date now = new Date();
                                if (updateTime.getTime() > now.getTime() - Constant.LAST_24_HOUR_TIME_STAMP) {
                                    Elements next_elements = document.select("a[class=\"nxt\"]");
                                    if(!next_elements.isEmpty()) {
                                        Element next_element = next_elements.get(0);
                                        String laterUrl = next_element.absUrl("href");
                                        if (!url.equals(laterUrl)) {
                                            haveLater = true;
                                            url = laterUrl;
                                        }
                                    }
                                }
                            }
                        }
                    }
                    //不存在下一页退出循环
                    if (!haveLater) {
                        break;
                    }
                } catch (Exception e) {
                    log.error(setting.getName() + "生成任务出错: ", e);
                }
            }
        }
    }

    /**
     * 抓天府社区回复的线程
     */
    private class TianFuReplyCrwler extends AbstractReplyCrawler {

        private ArrayList<Integer> pages;

        public TianFuReplyCrwler(ArrayList<Integer> pages, String url, String parentId, String ancestorId, Area area) {
            this.pages = pages;
            this.url = url;
            this.parentId = parentId;
            this.ancestorId = ancestorId;
            this.area = area;
        }

        @Override
        public List<SpeechBean> call() throws Exception {
            List<SpeechBean> reply_nodes = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                try {

                    String getUrl = url;
                    String lastPart = url.substring(0, getUrl.lastIndexOf("-"));
                    lastPart = lastPart.substring(0, lastPart.lastIndexOf("-")+1) + pages.get(k);
                    String laterPart = url.substring(getUrl.lastIndexOf("-"), url.length());
                    getUrl = lastPart + laterPart;
                    //String getUrl = url+"&page="+pages.get(k);
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(getUrl, false);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        SpeechBean reply = new SpeechBean();
                        Document document = entity.getDocument();
                        String title = document.title();
                        if(CommonUtil.isNull(title)){
                            String url1 = url + "?_dsign=" + getRealUrl(document.select("script").get(0).html());
                            entity = null;
                            try {
                                entity = downloader.get(url1, false);
                            } catch (Exception e) {
                                log.error(e.getMessage(), e);
                            }
                            if (entity != null && entity.getStatus_code() == 200) {
                                document = entity.getDocument();
                            }
                        }
                        document.select(".aimg_tip").remove();
                        Element pageInfo = null;
                        Elements replyInfos = null;
                        try {
                            pageInfo = document.getElementById("postlist");
                            replyInfos = pageInfo.select(".floor");
                        }catch (Exception e){
                            log.error(setting.getName() + "访问帖子的回复内容出错");
                            /*document = get(url,true);
                            pageInfo = document.getElementById("postlist");
                            replyInfos =  pageInfo.select(".floor");*/
                        }
                        for(int i =0;i<replyInfos.size();i++){
                            try{
                                Element replyInfo = replyInfos.get(i);
                                String reply_id = replyInfo.select("table").get(0).attr("id").substring(3);
                                String reply_user_home_url = replyInfo.select(".authi").select(".avatar").select("a").get(0).attr("href");
                                String reply_user_id = replyInfo.select(".authi").select(".avatar").select("a").get(0).attr("href").split("//")[1].split("-")[2].split("\\.")[0];
                                String reply_user_photo = replyInfo.select(".authi").select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                                String reply_publish_user =  replyInfo.select(".authi").select(".floor-profile").select("a").get(0).text();
                                String reply_publish_time = "";
                                if(replyInfo.select(".authi").select(".cp_pls").select("em")!=null){
                                    reply_publish_time = replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[1]+" "+replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[2]+":00";
                                }
                                String homeUrl = "http://bbs.scol.com.cn/forum.php?mod=viewthread&tid="+ancestorId+"&page="+pages.get(k)+"#pid"+reply_id;

                                String reply_content = replyInfo.select(".pcb").select(".t_fsz").select(".t_f").text();
                                List<String> reply_img_urls = new ArrayList<>();
                                Elements replyImgElements=replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select("img[zoomfile]");
                                for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                    Element imgElement=replyImgElements.get(imgIndex);
                                    String img_url=imgElement.absUrl("zoomfile");
                                    reply_img_urls.add(img_url);
                                }
                                reply.setId(reply_id);
                                reply.setPublish_user_id(reply_user_id);
                                reply.setPublish_user_photo(reply_user_photo);
                                reply.setPublish_user(reply_publish_user);
                                reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_publish_time));
                                reply.setUrl(homeUrl);
                                reply.setContent(reply_content);
                                reply.setImg_urls(reply_img_urls);
                                reply.setPublish_user_jsonid(setting.getName() + "_" + reply_user_id);
                                if (CommonUtil.isNotNull(reply_user_home_url)) {
                                    String userKey = setting.getName() + "_" + reply_user_id;
                                    Task hometask = new Task(reply_user_home_url, TaskType.USER, WebType.FORLUM, userKey);
                                    hometask.put("publish_user", reply_user_id);
                                    hometask.put("publish_user_photo", reply_user_photo);
                                    TaskManager.add(setting.getUser_task(), hometask);
                                }
                                reply.setParent_id(parentId);
                                reply.setAncestor_id(ancestorId);
                                reply.setArea(area);
                                reply_nodes.add(reply);
                            }catch (Exception e){
                                e.printStackTrace();//因为禁言无法获取的情况
                            }
                        }
                    }
                } catch (Exception e) {
                    log.error(setting.getName() + "帖子回复: " + url + "出错", e);
                }
            }
            return reply_nodes;
        }
    }

    // TODO: 2017/12/27 天府社区测试
    public static void main(String[] args) throws Exception {
        //new TianFuCrawler().generateTask();
        JSONObject object = new TianFuCrawler().crawlItem(new Task("http://bbs.scol.com.cn/thread-15195961-1-1.html", TaskType.SPEECH, WebType.FORLUM, "dd", null));
        if(CommonUtil.isNotNull(object)) {
            System.out.println(object.toJSONString());
        } else {
            System.exit(1);
        }

    }
}
